# -*- coding: utf-8 -*-
import scrapy
import re
import datetime
from scrapy.http import Request
from ArticleSpider.items import JobBoleArticleItem,ArticlItemLoad
from utils.common import get_md5
from scrapy.loader import ItemLoader

class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']

    def parse(self, response):
        '''
        1、获取文章列表中的文章url，并交给解析函数进行具体字段解析
        2、获取下一页url并交给scrapy进行下载，下载完成之后交给parse
        
        '''
        #post_urls=response.css(".grid-8>.floated-thumb .post-thumb a::attr(href)").extract()
        post_nodes=response.css(".grid-8>.floated-thumb .post-thumb a")
        for post_node in post_nodes:
            image_url=post_node.css("img::attr(src)").extract_first("")
            post_url=post_node.css("::attr(href)").extract_first("")
        #for post_url in post_urls:
            yield Request(url=post_url,meta={"front_image_url":image_url},callback=self.parse_detail)
        
        next_url=response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:
            yield Request(url=next_url,callback=self.parse)
    
    def parse_detail(self,response):  
        '''
        article_item=JobBoleArticleItem()
        
        front_image_url= response.meta["front_image_url"]
        #提取标题
        title=response.xpath('//div[@class="entry-header"]/h1/text()').extract()[0]
        #提取时间
        
        create_date=response.xpath('//p[@class="entry-meta-hide-on-mobile"]/text()').extract()[0].strip().replace("·","").strip()
            
        match_re=re.match(".*(\d+).*", response.xpath('//span[contains(@class,"bookmark-btn")]/text()').extract()[0])
        match_re_com=re.match(".*(\d+).*", response.xpath('//a[@href="#article-comment"]/span').extract_first("0")[0])
        #点赞数
        parise_num=response.xpath('//span[contains(@class,"vote-post-up")]/h10/text()').extract()[0]
        #收藏数
        fav_num=match_re.group(1) if match_re else 0
        #评论数
        com_num=match_re_com.group(1) if match_re_com else 0
        
        article_item["url_object_id"]=get_md5(response.url)
        article_item["title"]=title
        try:
            create_date=datetime.datetime.strptime(create_date,"%Y/%m/%d").data()
        except Exception as e:
            create_date=datetime.datetime.now().date()
        article_item["create_date"]=create_date
        article_item["parise_num"]=int(parise_num)
        article_item["fav_num"]=int(fav_num)
        article_item["com_num"]=int(com_num)
        article_item["front_image_url"]=[front_image_url]
        article_item["url"]=response.url
        '''
        
        #通过ItemLoader加载item
        #item_loader=ItemLoader(item=JobBoleArticleItem(),response=response)
        front_image_url = response.meta.get("front_image_url", "")  # 文章封面图
        item_loader = ArticlItemLoad(item=JobBoleArticleItem(), response=response)
        item_loader.add_css("title", ".entry-header h1::text")
        item_loader.add_value("url", response.url)
        item_loader.add_value("url_object_id", get_md5(response.url))
        #item_loader.add_css("create_date", "p.entry-meta-hide-on-mobile::text")
        item_loader.add_xpath("create_date",'//p[@class="entry-meta-hide-on-mobile"]/text()')
        item_loader.add_value("front_image_url", [front_image_url])
        item_loader.add_css("parise_num", ".vote-post-up h10::text")
        item_loader.add_css("com_num", "a[href='#article-comment'] span::text")
        item_loader.add_css("fav_num", ".bookmark-btn::text")
        item_loader.add_css("tags", "p.entry-meta-hide-on-mobile a::text")
        item_loader.add_css("content", "div.entry")
        article_item = item_loader.load_item()
        yield article_item
        
